{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'channels_last'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "\n",
    "\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "FLAGS = None\n",
    "K.image_data_format() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def swish(x):\n",
    "  return x * tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "#  return relu(x)\n",
    "#  return tf.nn.sigmoid(x)\n",
    "#  return tf.nn.elu(x)\n",
    "  return swish(x)\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=stddev)\n",
    "  #return tf.zeros(shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-7956cba52202>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/jck/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = 'tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "keep_prob = tf.placeholder(tf.float32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "L1_units_count = 100\n",
    "#x = tf.placeholder(tf.float32, [None, 784])\n",
    "#tf.shape(x)  [100, 784]\n",
    "# exponetial lr decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000, tf.shape(x)[0]))\n",
    "#计算多少个step是一个epoch\n",
    "#相当于是600个step就能把所有的图片每张跑上一次，就是600个是一个epoch，\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "#当前已经跑到了多少步\n",
    "current_epoch = global_step//epoch_steps #//600\n",
    "#当前已经跑到了多少个epoch\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.575, tf.to_float(decay_times)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第一个卷积层\n",
    "net = Conv2D(32, kernel_size=[5,5], strides=[1,1],activation=activation,\n",
    "                 padding='same',\n",
    "                input_shape=[28,28,1])(x_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第一个池化层\n",
    "net = MaxPooling2D(pool_size=[2,2])(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第二个卷积层\n",
    "net = Conv2D(64, kernel_size=[5,5], strides=[1,1],activation=activation,\n",
    "                padding='same')(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第二个池化层\n",
    "net = MaxPooling2D(pool_size=[2,2])(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#把第二个池化层的输出扁平化为1维\n",
    "net = Flatten()(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第一个全连接层\n",
    "net = Dense(1000, activation=activation)(net)\n",
    "#net = tf.nn.dropout(net,keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Dense(10,activation='softmax')(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.objectives import categorical_crossentropy\n",
    "cross_entropy = tf.reduce_mean(categorical_crossentropy(y_, net))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "K.set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 0.251705, l2_loss: 3862.896729, total loss: 0.522107\n",
      "0.94\n",
      "step 200, entropy loss: 0.158039, l2_loss: 2275.761475, total loss: 0.317342\n",
      "0.99\n",
      "step 300, entropy loss: 0.070654, l2_loss: 1740.749634, total loss: 0.192507\n",
      "0.98\n",
      "step 400, entropy loss: 0.184570, l2_loss: 1580.201294, total loss: 0.295184\n",
      "0.98\n",
      "step 500, entropy loss: 0.069675, l2_loss: 1304.871460, total loss: 0.161016\n",
      "0.98\n",
      "step 600, entropy loss: 0.122434, l2_loss: 1195.223877, total loss: 0.206099\n",
      "0.98\n",
      "step 700, entropy loss: 0.089052, l2_loss: 1022.846802, total loss: 0.160651\n",
      "0.98\n",
      "step 800, entropy loss: 0.033470, l2_loss: 922.848816, total loss: 0.098070\n",
      "1.0\n",
      "step 900, entropy loss: 0.088291, l2_loss: 837.203735, total loss: 0.146895\n",
      "0.99\n",
      "step 1000, entropy loss: 0.077575, l2_loss: 815.172119, total loss: 0.134637\n",
      "0.99\n",
      "0.9705\n",
      "step 1100, entropy loss: 0.086259, l2_loss: 857.484558, total loss: 0.146283\n",
      "0.99\n",
      "step 1200, entropy loss: 0.086379, l2_loss: 841.862671, total loss: 0.145309\n",
      "0.98\n",
      "step 1300, entropy loss: 0.110797, l2_loss: 726.415649, total loss: 0.161646\n",
      "0.98\n",
      "step 1400, entropy loss: 0.027458, l2_loss: 666.916260, total loss: 0.074143\n",
      "0.99\n",
      "step 1500, entropy loss: 0.007074, l2_loss: 612.216675, total loss: 0.049929\n",
      "1.0\n",
      "step 1600, entropy loss: 0.027331, l2_loss: 604.150024, total loss: 0.069622\n",
      "0.99\n",
      "step 1700, entropy loss: 0.010728, l2_loss: 579.814758, total loss: 0.051315\n",
      "1.0\n",
      "step 1800, entropy loss: 0.022556, l2_loss: 556.333984, total loss: 0.061500\n",
      "1.0\n",
      "step 1900, entropy loss: 0.040079, l2_loss: 520.338257, total loss: 0.076502\n",
      "0.98\n",
      "step 2000, entropy loss: 0.003292, l2_loss: 491.268097, total loss: 0.037681\n",
      "1.0\n",
      "0.9912\n",
      "step 2100, entropy loss: 0.009281, l2_loss: 470.487213, total loss: 0.042215\n",
      "1.0\n",
      "step 2200, entropy loss: 0.046866, l2_loss: 459.449524, total loss: 0.079027\n",
      "0.99\n",
      "step 2300, entropy loss: 0.003307, l2_loss: 433.717987, total loss: 0.033668\n",
      "1.0\n",
      "step 2400, entropy loss: 0.007571, l2_loss: 429.111664, total loss: 0.037608\n",
      "1.0\n",
      "step 2500, entropy loss: 0.004692, l2_loss: 411.536896, total loss: 0.033500\n",
      "1.0\n",
      "step 2600, entropy loss: 0.004396, l2_loss: 397.715820, total loss: 0.032236\n",
      "1.0\n",
      "step 2700, entropy loss: 0.008497, l2_loss: 391.945190, total loss: 0.035933\n",
      "1.0\n",
      "step 2800, entropy loss: 0.001131, l2_loss: 378.385101, total loss: 0.027618\n",
      "1.0\n",
      "step 2900, entropy loss: 0.001895, l2_loss: 364.041412, total loss: 0.027377\n",
      "1.0\n",
      "step 3000, entropy loss: 0.015230, l2_loss: 358.123444, total loss: 0.040299\n",
      "1.0\n",
      "0.991\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for step in range(3000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, init_learning_rate:lr})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(net, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
